{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([-0.03846154,  0.        , -0.53846154, -0.62222222])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import gym_pygame\n",
    "\n",
    "#!pip install git+https://github.com/ntasfi/PyGame-Learning-Environment.git\n",
    "#!pip install git+https://github.com/qlan3/gym-games.git\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Catcher-PLE-v0')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box(-inf, inf, (4,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [-0.03846154  0.         -0.07692308 -0.8       ]\n",
      "action= 2\n",
      "next_state= [-0.03846154  0.         -0.07692308 -0.73333333]\n",
      "reward= 0.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
      "  logger.deprecation(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
      "  logger.warn(f\"{pre} is not within the observation space.\")\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:272: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEWCAYAAACjTbhPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVfElEQVR4nO3df2xVd/3H8Ve73nv5Ue5tKXIvlV7AiKsTmVsZ5YpmidxI5qIDmskfM6IuLsAFgfnH0j9gkqiXbDEqBoc/Ipg4V60JU5ZMJGV02XJXaBcUBnbgSHpjey8ues8tjP5I7+f7h19PdkdLe+kH7i08H8k7WT/nc895c5a+cu45vfdTZowxAgCLyovdAIDbD8ECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwLqbFiz79u3TwoULNW3aNDU2NurEiRM361AASkzZzfis0O9+9zt99atf1f79+9XY2Kgf/ehHam1tVXd3t+bOnXvd1+ZyOfX29mrWrFkqKyuz3RqAG2SMUX9/v2pra1VePs41ibkJli9fbmKxmPvzyMiIqa2tNfF4fNzXJpNJI4miqBKtZDI57u+x9bdCQ0ND6urqUjQadcfKy8sVjUaVSCSumT84OKhsNuuW4cPWQEmbNWvWuHOsB8u7776rkZERBYPBvPFgMKhUKnXN/Hg8rkAg4FY4HLbdEgCLJnKLouhPhZqbm+U4jlvJZLLYLQGYpArbO5wzZ47uuusupdPpvPF0Oq1QKHTNfJ/PJ5/PZ7sNAEVk/YrF6/WqoaFBbW1t7lgul1NbW5sikYjtwwEoRZN5+jOWlpYW4/P5zMGDB83Zs2fNE088YaqqqkwqlRr3tY7jFP2uN0VRY5fjOOP+Hlt/KyRJ69ev17/+9S/t2rVLqVRKn/rUp/TnP//5mhu6AG5PN+UP5CYjm80qEAgUuw0AY3AcR36//7pziv5UCMDth2ABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwruBgefXVV/XFL35RtbW1Kisr04svvpi33RijXbt2ad68eZo+fbqi0ajOnz9vq18AU0DBwXLlyhXde++92rdv36jbn3nmGe3du1f79+9XR0eHZs6cqdWrV2tgYGDSzQKYIsZdNv46JJlDhw65P+dyORMKhcyzzz7rjmUyGePz+cwLL7wwoX06jjPhVe8pirr15TjOuL/HVu+xXLx4UalUStFo1B0LBAJqbGxUIpEY9TWDg4PKZrN5BWBqsxosqVRKkhQMBvPGg8Ggu+2D4vG4AoGAW3V1dTZbAlAERX8q1NzcLMdx3Eomk8VuCcAkWQ2WUCgkSUqn03nj6XTa3fZBPp9Pfr8/rwBMbVaDZdGiRQqFQmpra3PHstmsOjo6FIlEbB4KQAmrKPQFly9f1oULF9yfL168qFOnTmn27NkKh8Pavn27vvvd72rx4sVatGiRdu7cqdraWq1Zs8Zm3wBKWaGPmF955ZVRH0Ft2LDBfeS8c+dOEwwGjc/nM6tWrTLd3d0T3j+PmymqtGsij5vLjDFGJSSbzSoQCBS7DQBjcBxn3HuhRX8qBOD2Q7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYV1CwxONxPfDAA5o1a5bmzp2rNWvWqLu7O2/OwMCAYrGYampqVFlZqaamJqXTaatNAyhtBQVLe3u7YrGY3njjDR09elTDw8P6/Oc/rytXrrhzduzYocOHD6u1tVXt7e3q7e3VunXrrDcOoIQVuij8+126dMlIMu3t7cYYYzKZjPF4PKa1tdWdc+7cOSPJJBKJCe2TReEpqrRrIovCT+oei+M4kqTZs2dLkrq6ujQ8PKxoNOrOqa+vVzgcViKRGHUfg4ODymazeQVgarvhYMnlctq+fbtWrlypJUuWSJJSqZS8Xq+qqqry5gaDQaVSqVH3E4/HFQgE3Kqrq7vRlgCUiBsOllgspjNnzqilpWVSDTQ3N8txHLeSyeSk9geg+Cpu5EVbtmzRSy+9pFdffVXz5893x0OhkIaGhpTJZPKuWtLptEKh0Kj78vl88vl8N9IGgFJVyM3aXC5nYrGYqa2tNW+//fY12/938/YPf/iDO/b3v//dSNy8pajbpSZy87agYNm0aZMJBALm+PHjpq+vz6333nvPnbNx40YTDofNsWPHTGdnp4lEIiYSiUz4GAQLRZV2WQ+WsQ504MABd87Vq1fN5s2bTXV1tZkxY4ZZu3at6evrI1go6japiQRL2f8HRsnIZrMKBALFbgPAGBzHkd/vv+4cPisEwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwLqCguW5557T0qVL5ff75ff7FYlE9PLLL7vbBwYGFIvFVFNTo8rKSjU1NSmdTltvGkBpKyhY5s+frz179qirq0udnZ363Oc+p0ceeURvvfWWJGnHjh06fPiwWltb1d7ert7eXq1bt+6mNA6ghBWyKPxoqqurzS9/+UuTyWSMx+Mxra2t7rZz584ZSSaRSEx4fywKT1GlXRNZFP6G77GMjIyopaVFV65cUSQSUVdXl4aHhxWNRt059fX1CofDSiQSY+5ncHBQ2Ww2rwBMbQUHy+nTp1VZWSmfz6eNGzfq0KFDuueee5RKpeT1elVVVZU3PxgMKpVKjbm/eDyuQCDgVl1dXcH/CAClpeBgufvuu3Xq1Cl1dHRo06ZN2rBhg86ePXvDDTQ3N8txHLeSyeQN7wtAaago9AVer1cf/ehHJUkNDQ06efKkfvzjH2v9+vUaGhpSJpPJu2pJp9MKhUJj7s/n88nn8xXeOYCSNem/Y8nlchocHFRDQ4M8Ho/a2trcbd3d3erp6VEkEpnsYQBMIQVdsTQ3N+uhhx5SOBxWf3+/fvvb3+r48eM6cuSIAoGAHn/8cT355JOaPXu2/H6/tm7dqkgkohUrVtys/gGUokIeLX/jG98wCxYsMF6v13zoQx8yq1atMn/5y1/c7VevXjWbN2821dXVZsaMGWbt2rWmr6+vkEPwuJmiSrwm8ri5zBhjVEKy2awCgUCx2wAwBsdx5Pf7rzuHzwoBsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsG5SwbJnzx6VlZVp+/bt7tjAwIBisZhqampUWVmppqYmpdPpyfYJYAq54WA5efKkfvazn2np0qV54zt27NDhw4fV2tqq9vZ29fb2at26dZNuFMAUUtCK7f+vv7/fLF682Bw9etQ8+OCDZtu2bcYYYzKZjPF4PKa1tdWde+7cOSPJJBKJCe2bReEpqrRrIovC39AVSywW08MPP6xoNJo33tXVpeHh4bzx+vp6hcNhJRKJUfc1ODiobDabVwCmtopCX9DS0qI333xTJ0+evGZbKpWS1+tVVVVV3ngwGFQqlRp1f/F4XLt37y60DQAlrKArlmQyqW3btun555/XtGnTrDTQ3Nwsx3HcSiaTVvYLoHgKCpauri5dunRJ999/vyoqKlRRUaH29nbt3btXFRUVCgaDGhoaUiaTyXtdOp1WKBQadZ8+n09+vz+vAExtBb0VWrVqlU6fPp039vWvf1319fV66qmnVFdXJ4/Ho7a2NjU1NUmSuru71dPTo0gkYq9rACWtoGCZNWuWlixZkjc2c+ZM1dTUuOOPP/64nnzySc2ePVt+v19bt25VJBLRihUr7HUNoKQVfPN2PD/84Q9VXl6upqYmDQ4OavXq1frpT39q+zAASliZMcYUu4n3y2azCgQCxW4DwBgcxxn3XiifFQJgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgXUHB8p3vfEdlZWV5VV9f724fGBhQLBZTTU2NKisr1dTUpHQ6bb1pAKWt4CuWT3ziE+rr63Prtddec7ft2LFDhw8fVmtrq9rb29Xb26t169ZZbRjAFGAK8PTTT5t777131G2ZTMZ4PB7T2trqjp07d85IMolEYsLHcBzHSKIoqkTLcZxxf48LvmI5f/68amtr9ZGPfESPPfaYenp6JEldXV0aHh5WNBp159bX1yscDiuRSBR6GABTWEUhkxsbG3Xw4EHdfffd6uvr0+7du/XZz35WZ86cUSqVktfrVVVVVd5rgsGgUqnUmPscHBzU4OCg+3M2my3sXwCg5BQULA899JD730uXLlVjY6MWLFig3//+95o+ffoNNRCPx7V79+4bei2A0jSpx81VVVX62Mc+pgsXLigUCmloaEiZTCZvTjqdVigUGnMfzc3NchzHrWQyOZmWAJSASQXL5cuX9Y9//EPz5s1TQ0ODPB6P2tra3O3d3d3q6elRJBIZcx8+n09+vz+vAExxE35cY4z59re/bY4fP24uXrxoXn/9dRONRs2cOXPMpUuXjDHGbNy40YTDYXPs2DHT2dlpIpGIiUQihRyCp0IUVeI1kadCBQXL+vXrzbx584zX6zUf/vCHzfr1682FCxfc7VevXjWbN2821dXVZsaMGWbt2rWmr6+PYKGo26gmEixlxhijEpLNZhUIBIrdBoAxOI4z7i0LPisEwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYVtBIibi8l9j3qBSkrKyt2C7gOrlgAWEewALCOYAFgHcECwDqCBYB1BQfLP//5T33lK19RTU2Npk+frk9+8pPq7Ox0txtjtGvXLs2bN0/Tp09XNBrV+fPnrTYNoLQVFCz/+c9/tHLlSnk8Hr388ss6e/asfvCDH6i6utqd88wzz2jv3r3av3+/Ojo6NHPmTK1evVoDAwPWmwdQosZdNv59nnrqKfOZz3xmzO25XM6EQiHz7LPPumOZTMb4fD7zwgsvTOgYjuNMeNV7anI1lRX73N3J5TjOuP9/Crpi+dOf/qRly5bp0Ucf1dy5c3XffffpF7/4hbv94sWLSqVSikaj7lggEFBjY6MSicSo+xwcHFQ2m80rAFNbQcHyzjvv6LnnntPixYt15MgRbdq0Sd/61rf061//WpKUSqUkScFgMO91wWDQ3fZB8XhcgUDArbq6uhv5dwAoJYVcfno8HhOJRPLGtm7dalasWGGMMeb11183kkxvb2/enEcffdR8+ctfHnWfAwMDxnEct5LJZNEv9SiKGrusvxWaN2+e7rnnnryxj3/84+rp6ZEkhUIhSVI6nc6bk06n3W0f5PP55Pf78wrA1FZQsKxcuVLd3d15Y2+//bYWLFggSVq0aJFCoZDa2trc7dlsVh0dHYpEIhbaBTAlFPJW6MSJE6aiosJ873vfM+fPnzfPP/+8mTFjhvnNb37jztmzZ4+pqqoyf/zjH83f/vY388gjj5hFixaZq1evTugYPBWiqNKuibwVKviZ4+HDh82SJUuMz+cz9fX15uc//3ne9lwuZ3bu3GmCwaDx+Xxm1apVpru7e8L7J1goqrRrIsFSZkxpfSlHNptVIBAodhsAxuA4zrj3QvmsEADrCBYA1hEsAKwjWABYV3LBUmL3kgF8wER+R0suWPr7+4vdAoDrmMjvaMk9bs7lcurt7ZUxRuFwWMlkkj/zH0U2m1VdXR3n5zo4R9dX6Pkxxqi/v1+1tbUqL7/+NUnJrStUXl6u+fPnu1+fwOeHro/zMz7O0fUVcn4m+jdmJfdWCMDUR7AAsK5kg8Xn8+npp5+Wz+crdislifMzPs7R9d3M81NyN28BTH0le8UCYOoiWABYR7AAsI5gAWBdyQbLvn37tHDhQk2bNk2NjY06ceJEsVsqing8rgceeECzZs3S3LlztWbNmmu+d3hgYECxWEw1NTWqrKxUU1PTNV9ofifYs2ePysrKtH37dneMc1OkZZEn/J2Rt1BLS4vxer3mV7/6lXnrrbfMN7/5TVNVVWXS6XSxW7vlVq9ebQ4cOGDOnDljTp06Zb7whS+YcDhsLl++7M7ZuHGjqaurM21tbaazs9OsWLHCfPrTny5i17feiRMnzMKFC83SpUvNtm3b3PE7/dz8+9//NgsWLDBf+9rXTEdHh3nnnXfMkSNHzIULF9w5e/bsMYFAwLz44ovmr3/9q/nSl75U0PdUj6Ykg2X58uUmFou5P4+MjJja2loTj8eL2FVpuHTpkpFk2tvbjTH/XcLW4/GY1tZWd865c+eMJJNIJIrV5i3V399vFi9ebI4ePWoefPBBN1g4N7dmWeTRlNxboaGhIXV1deUt01peXq5oNDrmMq13EsdxJEmzZ8+WJHV1dWl4eDjvfNXX1yscDt8x5ysWi+nhhx/OOwcS50a6OcsiT0TJBcu7776rkZGRgpZpvVPkcjlt375dK1eu1JIlSyT9d1lbr9erqqqqvLl3yvlqaWnRm2++qXg8fs22O/3cSDdnWeSJKLlPN2NssVhMZ86c0WuvvVbsVkpCMpnUtm3bdPToUU2bNq3Y7ZSkXC6nZcuW6fvf/74k6b777tOZM2e0f/9+bdiw4aYdt+SuWObMmaO77rqroGVa7wRbtmzRSy+9pFdeeUXz5893x0OhkIaGhpTJZPLm3wnnq6urS5cuXdL999+viooKVVRUqL29XXv37lVFRYWCweAde27+52YsizwRJRcsXq9XDQ0Necu05nI5tbW13ZHLtBpjtGXLFh06dEjHjh3TokWL8rY3NDTI4/Hkna/u7m719PTc9udr1apVOn36tE6dOuXWsmXL9Nhjj7n/faeem/8p2rLIN3zb9yZqaWkxPp/PHDx40Jw9e9Y88cQTpqqqyqRSqWK3dstt2rTJBAIBc/z4cdPX1+fWe++9587ZuHGjCYfD5tixY6azs9NEIhETiUSK2HXxvP+pkDGcm1uxLPJoSjJYjDHmJz/5iQmHw8br9Zrly5ebN954o9gtFYXGWObywIED7pyrV6+azZs3m+rqajNjxgyzdu1a09fXV7ymi+iDwcK5ufnLIo+Gr00AYF3J3WMBMPURLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALDu/wC0i9mj5k1gzAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7fefed1342e0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-7.0, 1.140175425099138)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1fa74b09d19451cacd5dabea660cf75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">WARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">rewrite the environment with new step API. </span>\n",
       "  logger.deprecation(\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \n",
       "\u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to \u001b[0m\n",
       "\u001b[33mrewrite the environment with new step API. \u001b[0m\n",
       "  logger.deprecation(\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: <span style=\"color: #808000; text-decoration-color: #808000\">WARN: The </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64</span>\n",
       "  logger.warn(\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The \u001b[0m\n",
       "\u001b[33mobs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
       "  logger.warn(\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: <span style=\"color: #808000; text-decoration-color: #808000\">WARN: The </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">obs returned by the `step()` method is not within the observation space.</span>\n",
       "  logger.warn(f\"{pre} is not within the observation space.\")\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The \u001b[0m\n",
       "\u001b[33mobs returned by the `step()` method is not within the observation space.\u001b[0m\n",
       "  logger.warn(f\"{pre} is not within the observation space.\")\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/1.PPO.Catcher')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11.3, 0.45825756949558394)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/1.PPO.Catcher')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEWCAYAAACjTbhPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVg0lEQVR4nO3df2xVd/3H8VdL7738KPeWdnAvDb2AEVcnMrcyypWZJXKVzGUOaOayzIhKXIAL8iMms38Akqi32WKmGDb8EcHEuWpNmLJkIimjy8ylQBcUBnbgSHq1vRen3nsLo7cN9/P9w68n3tFCL/3AvYXnI3kn6+d8zrlvztJXzj3n3n7KjDFGAGBRebEbAHD7IVgAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWHfTgmXXrl2aM2eOJk6cqMbGRh09evRmvRSAElN2M74r9Ktf/Upf+tKXtHv3bjU2Nur73/++2tra1N3drRkzZlxz31wup97eXk2dOlVlZWW2WwNwg4wx6u/vV21trcrLr3NNYm6CRYsWmUgk4vx85coVU1tba6LR6HX3jcfjRhJFUSVa8Xj8ur/H1t8KDQ4OqqurS+Fw2BkrLy9XOBxWLBa7an42m1Umk3HK8GVroKRNnTr1unOsB8t7772nK1euyO/35437/X4lEomr5kejUfl8PqeCwaDtlgBYNJpbFEV/KtTc3Kx0Ou1UPB4vdksAxqjC9gHvuusuTZgwQclkMm88mUwqEAhcNd/j8cjj8dhuA0ARWb9icbvdamhoUHt7uzOWy+XU3t6uUChk++UAlKKxPP0ZSWtrq/F4PGbv3r3m9OnT5umnnzZVVVUmkUhcd990Ol30u94URY1c6XT6ur/H1t8KSdITTzyhf/zjH9q2bZsSiYQ+8YlP6Pe///1VN3QB3J5uygfkxiKTycjn8xW7DQAjSKfT8nq915xT9KdCAG4/BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BQfLG2+8oUcffVS1tbUqKyvTK6+8krfdGKNt27Zp5syZmjRpksLhsM6ePWurXwDjQMHBcunSJd17773atWvXsNufffZZ7dy5U7t371ZnZ6emTJmiZcuWaWBgYMzNAhgnrrts/DVIMvv27XN+zuVyJhAImOeee84ZS6VSxuPxmJdffnlUx0yn06Ne9Z6iqFtf6XT6ur/HVu+xnD9/XolEQuFw2Bnz+XxqbGxULBYbdp9sNqtMJpNXAMY3q8GSSCQkSX6/P2/c7/c72z4oGo3K5/M5VVdXZ7MlAEVQ9KdCzc3NSqfTTsXj8WK3BGCMrAZLIBCQJCWTybzxZDLpbPsgj8cjr9ebVwDGN6vBMnfuXAUCAbW3tztjmUxGnZ2dCoVCNl8KQAmrKHSHixcv6ty5c87P58+f14kTJ1RdXa1gMKhNmzbp29/+tubNm6e5c+dq69atqq2t1fLly232DaCUFfqI+fXXXx/2EdSqVaucR85bt241fr/feDwes3TpUtPd3T3q4/O4maJKu0bzuLnMGGNUQjKZjHw+X7HbADCCdDp93XuhRX8qBOD2Q7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYV1CwRKNRPfDAA5o6dapmzJih5cuXq7u7O2/OwMCAIpGIampqVFlZqaamJiWTSatNAyhtBQVLR0eHIpGIjhw5ooMHD2poaEif/exndenSJWfO5s2btX//frW1tamjo0O9vb1auXKl9cYBlLBCF4X/XxcuXDCSTEdHhzHGmFQqZVwul2lra3PmnDlzxkgysVhsVMdkUXiKKu0azaLwY7rHkk6nJUnV1dWSpK6uLg0NDSkcDjtz6uvrFQwGFYvFhj1GNptVJpPJKwDj2w0HSy6X06ZNm7RkyRLNnz9fkpRIJOR2u1VVVZU31+/3K5FIDHucaDQqn8/nVF1d3Y22BKBE3HCwRCIRnTp1Sq2trWNqoLm5Wel02ql4PD6m4wEovoob2Wn9+vV69dVX9cYbb2jWrFnOeCAQ0ODgoFKpVN5VSzKZVCAQGPZYHo9HHo/nRtoAUKoKuVmby+VMJBIxtbW15p133rlq+39v3v7mN79xxv7yl78YiZu3FHW71Ghu3hYULGvXrjU+n88cPnzY9PX1OfX+++87c9asWWOCwaA5dOiQOX78uAmFQiYUCo36NQgWiirtsh4sI73Qnj17nDmXL18269atM9OmTTOTJ082K1asMH19fQQLRd0mNZpgKfv/wCgZmUxGPp+v2G0AGEE6nZbX673mHL4rBMA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsC6goLlxRdf1IIFC+T1euX1ehUKhfTaa6852wcGBhSJRFRTU6PKyko1NTUpmUxabxpAaSsoWGbNmqWWlhZ1dXXp+PHj+vSnP63HHntMb7/9tiRp8+bN2r9/v9ra2tTR0aHe3l6tXLnypjQOoIQVsij8cKZNm2Z++tOfmlQqZVwul2lra3O2nTlzxkgysVhs1MdjUXiKKu0azaLwN3yP5cqVK2ptbdWlS5cUCoXU1dWloaEhhcNhZ059fb2CwaBisdiIx8lms8pkMnkFYHwrOFhOnjypyspKeTwerVmzRvv27dM999yjRCIht9utqqqqvPl+v1+JRGLE40WjUfl8Pqfq6uoK/kcAKC0FB8vdd9+tEydOqLOzU2vXrtWqVat0+vTpG26gublZ6XTaqXg8fsPHAlAaKgrdwe1268Mf/rAkqaGhQceOHdMPfvADPfHEExocHFQqlcq7akkmkwoEAiMez+PxyOPxFN45gJI15s+x5HI5ZbNZNTQ0yOVyqb293dnW3d2tnp4ehUKhsb4MgHGkoCuW5uZmPfzwwwoGg+rv79cvf/lLHT58WAcOHJDP59Pq1au1ZcsWVVdXy+v1asOGDQqFQlq8ePHN6h9AKSrk0fJXv/pVM3v2bON2u8306dPN0qVLzR/+8Adn++XLl826devMtGnTzOTJk82KFStMX19fIS/B42aKKvEazePmMmOMUQnJZDLy+XzFbgPACNLptLxe7zXn8F0hANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANaNKVhaWlpUVlamTZs2OWMDAwOKRCKqqalRZWWlmpqalEwmx9ongHHkhoPl2LFj+tGPfqQFCxbkjW/evFn79+9XW1ubOjo61Nvbq5UrV465UQDjSEErtv+//v5+M2/ePHPw4EHz0EMPmY0bNxpjjEmlUsblcpm2tjZn7pkzZ4wkE4vFRnVsFoWnqNKu0SwKf0NXLJFIRI888ojC4XDeeFdXl4aGhvLG6+vrFQwGFYvFhj1WNptVJpPJKwDjW0WhO7S2tuqtt97SsWPHrtqWSCTkdrtVVVWVN+73+5VIJIY9XjQa1Y4dOwptA0AJK+iKJR6Pa+PGjXrppZc0ceJEKw00NzcrnU47FY/HrRwXQPEUFCxdXV26cOGC7r//flVUVKiiokIdHR3auXOnKioq5Pf7NTg4qFQqlbdfMplUIBAY9pgej0derzevAIxvBb0VWrp0qU6ePJk39pWvfEX19fV65plnVFdXJ5fLpfb2djU1NUmSuru71dPTo1AoZK9rACWtoGCZOnWq5s+fnzc2ZcoU1dTUOOOrV6/Wli1bVF1dLa/Xqw0bNigUCmnx4sX2ugZQ0gq+eXs9zz//vMrLy9XU1KRsNqtly5bphRdesP0yAEpYmTHGFLuJ/5XJZOTz+YrdBoARpNPp694L5btCAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwrKFi+9a1vqaysLK/q6+ud7QMDA4pEIqqpqVFlZaWampqUTCatNw2gtBV8xfKxj31MfX19Tr355pvOts2bN2v//v1qa2tTR0eHent7tXLlSqsNAxgHTAG2b99u7r333mG3pVIp43K5TFtbmzN25swZI8nEYrFRv0Y6nTaSKIoq0Uqn09f9PS74iuXs2bOqra3Vhz70IT311FPq6emRJHV1dWloaEjhcNiZW19fr2AwqFgsVujLABjHKgqZ3NjYqL179+ruu+9WX1+fduzYoU996lM6deqUEomE3G63qqqq8vbx+/1KJBIjHjObzSqbzTo/ZzKZwv4FAEpOQcHy8MMPO/+9YMECNTY2avbs2fr1r3+tSZMm3VAD0WhUO3bsuKF9AZSmgoLlg6qqqvSRj3xE586d02c+8xkNDg4qlUrlXbUkk0kFAoERj9Hc3KwtW7Y4P2cyGdXV1Y2lLZQQ881v3vC+ZS0tFjvBrTSmz7FcvHhRf/3rXzVz5kw1NDTI5XKpvb3d2d7d3a2enh6FQqERj+HxeOT1evMKwPhW0BXLN77xDT366KOaPXu2ent7tX37dk2YMEFPPvmkfD6fVq9erS1btqi6ulper1cbNmxQKBTS4sWLb1b/AEpQQcHyt7/9TU8++aT++c9/avr06XrwwQd15MgRTZ8+XZL0/PPPq7y8XE1NTcpms1q2bJleeOGFm9I4gNJVZowxxW7if2UyGfl8vmK3AUu4x3L7SafT171lwXeFAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjo/0AygIH+kHUBQECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHVjWrAMt68S+0B2QcrKyordwh2PKxYA1hEsAKwjWABYR7AAsI5gAWBdwcHy97//XV/84hdVU1OjSZMm6eMf/7iOHz/ubDfGaNu2bZo5c6YmTZqkcDiss2fPWm0aQGkrKFj+/e9/a8mSJXK5XHrttdd0+vRpfe9739O0adOcOc8++6x27typ3bt3q7OzU1OmTNGyZcs0MDBgvXkAJcoU4JlnnjEPPvjgiNtzuZwJBALmueeec8ZSqZTxeDzm5ZdfHtVrpNNpI4kqco1nxT53t3ul0+nr/j8o6Irld7/7nRYuXKjHH39cM2bM0H333aef/OQnzvbz588rkUgoHA47Yz6fT42NjYrFYsMeM5vNKpPJ5BWA8a2gYHn33Xf14osvat68eTpw4IDWrl2rr3/96/r5z38uSUokEpIkv9+ft5/f73e2fVA0GpXP53Oqrq7uRv4dAEpJIZeYLpfLhEKhvLENGzaYxYsXG2OM+eMf/2gkmd7e3rw5jz/+uPnCF74w7DEHBgZMOp12Kh6PF/1Sj6Kokcv6W6GZM2fqnnvuyRv76Ec/qp6eHklSIBCQJCWTybw5yWTS2fZBHo9HXq83rwCMbwUFy5IlS9Td3Z039s4772j27NmSpLlz5yoQCKi9vd3Znslk1NnZqVAoZKFdAONCIW+Fjh49aioqKsx3vvMdc/bsWfPSSy+ZyZMnm1/84hfOnJaWFlNVVWV++9vfmj//+c/mscceM3PnzjWXL18e1WvwVIiiSrtG81ao4OeK+/fvN/Pnzzcej8fU19ebH//4x3nbc7mc2bp1q/H7/cbj8ZilS5ea7u7uUR+fYKGo0q7RBAsrIQIoCCshAigKggWAdQQLAOsIFgDWlVywlNi9ZAAfMJrf0ZILlv7+/mK3AOAaRvM7WnKPm3O5nHp7e2WMUTAYVDwe52P+w8hkMqqrq+P8XAPn6NoKPT/GGPX396u2tlbl5de+Jim5dYXKy8s1a9Ys588n8P2ha+P8XB/n6NoKOT+j/YxZyb0VAjD+ESwArCvZYPF4PNq+fbs8Hk+xWylJnJ/r4xxd2808PyV38xbA+FeyVywAxi+CBYB1BAsA6wgWANaVbLDs2rVLc+bM0cSJE9XY2KijR48Wu6WiiEajeuCBBzR16lTNmDFDy5cvv+rvDg8MDCgSiaimpkaVlZVqamq66g+a3wlaWlpUVlamTZs2OWOcmyItizzqvxl5C7W2thq3221+9rOfmbffftt87WtfM1VVVSaZTBa7tVtu2bJlZs+ePebUqVPmxIkT5nOf+5wJBoPm4sWLzpw1a9aYuro6097ebo4fP24WL15sPvnJTxax61vv6NGjZs6cOWbBggVm48aNzvidfm7+9a9/mdmzZ5svf/nLprOz07z77rvmwIED5ty5c86clpYW4/P5zCuvvGL+9Kc/mc9//vMF/Z3q4ZRksCxatMhEIhHn5ytXrpja2loTjUaL2FVpuHDhgpFkOjo6jDH/WcLW5XKZtrY2Z86ZM2eMJBOLxYrV5i3V399v5s2bZw4ePGgeeughJ1g4N7dmWeThlNxbocHBQXV1deUt01peXq5wODziMq13knQ6LUmqrq6WJHV1dWloaCjvfNXX1ysYDN4x5ysSieiRRx7JOwcS50a6Ocsij0bJBct7772nK1euFLRM650il8tp06ZNWrJkiebPny/pP8vaut1uVVVV5c29U85Xa2ur3nrrLUWj0au23ennRro5yyKPRsl9uxkji0QiOnXqlN58881it1IS4vG4Nm7cqIMHD2rixInFbqck5XI5LVy4UN/97nclSffdd59OnTql3bt3a9WqVTftdUvuiuWuu+7ShAkTClqm9U6wfv16vfrqq3r99dc1a9YsZzwQCGhwcFCpVCpv/p1wvrq6unThwgXdf//9qqioUEVFhTo6OrRz505VVFTI7/ffsefmv27GssijUXLB4na71dDQkLdMay6XU3t7+x25TKsxRuvXr9e+fft06NAhzZ07N297Q0ODXC5X3vnq7u5WT0/PbX++li5dqpMnT+rEiRNOLVy4UE899ZTz33fqufmvoi2LfMO3fW+i1tZW4/F4zN69e83p06fN008/baqqqkwikSh2a7fc2rVrjc/nM4cPHzZ9fX1Ovf/++86cNWvWmGAwaA4dOmSOHz9uQqGQCYVCRey6eP73qZAxnJtbsSzycEoyWIwx5oc//KEJBoPG7XabRYsWmSNHjhS7paLQCMtc7tmzx5lz+fJls27dOjNt2jQzefJks2LFCtPX11e8povog8HCubn5yyIPhz+bAMC6krvHAmD8I1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFg3f8Bsc7wtrLMVbsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.0 400 [0.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 3 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
